from urllib.parse import urlencode
import pandas
import datacompy


path1 = 'C:/zhijue/rb/rb_crawl_inside_tb/cldownload/2022-02-23/4088/xxx.xlsx'
path2 = 'C:/zhijue/rb/rb_crawl_inside_tb/cldownload/2022-02-24/4088/xxx.xlsx'


def drop_dna(df1, df2):
    len_df = len(df1)
    # print(df1)
    # print(df2)
    # 将含有大量0的列删除
    for index, row in df1.iteritems():
        x = 0
        for value in row:
            if value == 0.0:
                x += 1
        print(x)
        if x > len_df / 2:
            df1 = df1.drop([index], axis=1)
            df2 = df2.drop([index], axis=1)
    drop_index1 = []
    drop_index2 = []
    # 这一句是主要问题，把df格式转化为list，这样就可以遍历处理了
    # 删除含有0或者空值的行
    iterlist1 = list(df1.itertuples(index=True))
    for index, col in enumerate(iterlist1):
        if '-' in col or '0' in col:
            drop_index1.append(index)
    iterlist2 = list(df2.itertuples(index=True))
    for index, col in enumerate(iterlist2):
        if '-' in col or '0' in col:
            drop_index2.append(index)
    df1 = df1.drop(df1.index[drop_index1])
    df2 = df2.drop(df2.index[drop_index2])
    df1 = df1.dropna(axis=0)
    df2 = df2.dropna(axis=0)
    # print(df1)
    # print(df2)
    columns = len(df1.columns)
    print(columns)
    return df1, df2, columns


def pandas_compare2(f1, f2):
    try:
        df1 = pandas.read_excel(f1)
        df2 = pandas.read_excel(f2)
    except:
        df1 = pandas.read_csv(f1)
        df2 = pandas.read_csv(f2)
    df1, df2, df_columns = drop_dna(df1, df2)
    # 针对不同长度字段的文件，设置相似度要求
    if df_columns > 30:
        check_num = 20
    elif df_columns > 20:
        check_num = 10
    elif df_columns > 8:
        check_num = 6
    else:
        check_num = 4
    final_list = []
    try:
        df3 = df1.eq(df2)
        len_df = len(df3)
        for index, row in df3.iteritems():
            x = 0
            for value in row:
                if value is True:
                    x += 1
            # print(x)
            if x > len_df/4:
                final_list.append(True)
        print(final_list)
        if len(final_list) > check_num:
            return True
        else:
            return False
    except Exception as err:
        print(err)


print(pandas_compare2(path1, path2))

